package com.cnblogs;
 
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.ConnectException;
import java.util.HashMap;
import java.util.Map;

import org.apache.log4j.LogManager;
import org.jsoup.*;
import org.jsoup.nodes.*;
import org.jsoup.select.*;

import com.csdn.GetFileName;
import com.csdn.HttpUtils;
import com.csdn.UserAgentMiddle;
 

 
public class Main {

	private static org.apache.log4j.Logger log = LogManager.getLogger(Main.class);
 
	private static final String expertsURl = "http://www.cnblogs.com/AllBloggers.aspx";
	private static final int settime = 300000;
 

	public static void main(String[] args) throws IOException {

		//getTag("http://www.cnblogs.com/fuge/");
		//getExperts("http://www.cnblogs.com/tintown/category/210885.html");
		//getTopic("http://www.cnblogs.com/tintown/archive/2010/04/21/1717609.html");
		
		System.setProperty("http.maxRedirects", "50");
		System.getProperties().setProperty("proxySet", "true");
		System.getProperties().setProperty("http.proxyHost", "183.222.102.105");
		System.getProperties().setProperty("http.proxyPort", "80");	  
		index();

	}

	public static void index() throws IOException {
		Connection getAllExperts = Jsoup
				.connect(expertsURl)
				.userAgent(
						"Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:50.0) Gecko/20100101 Firefox/50.0")
				.timeout(settime).method(Connection.Method.GET);

		Document ex = getAllExperts.get();
		Element exbody = ex.body();
		int count = 0;
		Elements linkNodes = exbody.getElementsByTag("td");
		for (Element el : linkNodes) {
			Elements el3 = el.select("a");
			if (!el3.isEmpty()) {
				count++;
				Element linkNode = (el3.select("a")).get(0);
				String url = linkNode.attr("href");
				log.info("正在爬第："+count +"博客："+url);
				try {
					getTag(url);
				} catch (Exception e) {
					e.printStackTrace();
					log.error("没爬:"+url +"错误原因:"+e.getMessage());
					continue;
				}
			}

		}
	}
	
	// 标签
	public static  void getTag(String url) throws IOException{
		String userName = url.substring(0,url.length()-1);
		userName = userName.substring(userName.lastIndexOf("/")+1,userName.length());
		Connection getAllExpert = Jsoup
				.connect("http://www.cnblogs.com/"+userName+"/mvc/blog/sidecolumn.aspx?blogApp="+userName)
				.userAgent(UserAgentMiddle.Agent()).timeout(3000)
				.method(Connection.Method.GET);
		Document exe = getAllExpert.get();
		Element ebody = exe.body();
		Element es  = ebody.getElementById("sidebar_categories");
		Element e = es.getElementById("cell");
		Elements els = null;
		if(e== null){
			Elements ess = es.getElementsByClass("catListPostCategory");
			if(ess.isEmpty()){
				ess = es.getElementsByClass("catList");
			}
			if(ess != null ){
				els = ess.get(0).select("li a");
			}
			
		}else{
			  els = e.select("li a");
		}
		if(els .isEmpty()){
			throw new RuntimeException("标签为空:"+ url);
		}
		int rest = 0;
		for (Element element : els) {
			rest ++;
			if(rest==2){
				System.setProperty("http.maxRedirects", "50");
				System.getProperties().setProperty("proxySet", "true");
				System.getProperties().setProperty("http.proxyHost", "183.222.102.101");
				System.getProperties().setProperty("http.proxyPort", "80");	
			}else if(rest ==4){
				System.setProperty("http.maxRedirects", "50");
				System.getProperties().setProperty("proxySet", "true");
				System.getProperties().setProperty("http.proxyHost", "106.119.0.244");
				System.getProperties().setProperty("http.proxyPort", "8080");	
				rest = 0;
			}
			
			String href = element.attr("href");
			String tag = GetFileName.unicode(   element.text());
			log.info("爬取标签:"+tag +":"+href);
			try {
				getExperts(href,tag,userName);
			} catch (ConnectException e1) {// 超时重试
				e1.printStackTrace();
				log.info(tagRest+":重试爬取标签:"+tag +":"+href);
				tagRest ++;
				System.setProperty("http.maxRedirects", "50");
				System.getProperties().setProperty("proxySet", "true");
				System.getProperties().setProperty("http.proxyHost", "183.222.102.107");
				System.getProperties().setProperty("http.proxyPort", "80");	 
				getExperts(href,tag,userName);
				if(tagRest == 2){
					log.info(tagRest+":重试爬取标签:"+tag +":"+href);
					System.setProperty("http.maxRedirects", "50");
					System.getProperties().setProperty("proxySet", "true");
					System.getProperties().setProperty("http.proxyHost", "183.222.102.101");
					System.getProperties().setProperty("http.proxyPort", "80");
					getExperts(href,tag,userName);
					tagRest= 0;
				}
				
				
			} catch (RuntimeException e2) {
				 log.info(e2.getMessage());
				 continue;
			}
			
		}
	}
	static int tagRest = 0;
	

	// 文章列表
	public static void getExperts(String url,String tag,String userName) throws IOException {
		Connection conn = Jsoup.connect(url).userAgent(UserAgentMiddle.Agent())
				.timeout(settime).method(Connection.Method.GET);
		Document doc = conn.get();
		Element body = doc.body();
		Elements els = body.getElementsByClass("posthead");
		if(els.isEmpty()){
			 els = body.getElementsByClass("entrylistItemTitle");
		}
		
		if(els.isEmpty()){
			 els = body.getElementsByClass("PostTitle");
		}
	 
		if(els.isEmpty()){
			throw new RuntimeException("文章列表为空:"+ url +"tag:"+tag);
		}
		
		for (Element element : els) {
			Element el = element.select("a").get(0);
			String href = el.attr("href");
			String title = el.text();
			String topic = getTopic(href);
			if(topic == null){
				continue;
			}
			connectionUrl(title, topic, userName, 1, 1, 1, "fxbbs", tag);
		}
		
	}

	// 获取他的博客内容
	public static String getTopic(String url) throws IOException {

		Connection conn = Jsoup.connect(url).userAgent(UserAgentMiddle.Agent())
				.timeout(settime).method(Connection.Method.GET);
		Document doc = conn.get();
		Element body = doc.body();
		Element html = body.getElementById("cnblogs_post_body");
		if(html == null){
			return null;
		}
		return html.toString();
	}

	public static void connectionUrl(String title, String content,
			String userName, Integer forumId, Integer postTypeId,
			Integer category, String ps,String fourmName) throws UnsupportedEncodingException {
		Map<String, String> params = new HashMap<String, String>();
		params.put("title", title);
		params.put("content", content);
		params.put("userName", userName);
		params.put("forumId", forumId + "");
		params.put("postTypeId", postTypeId + "");
		params.put("ps", ps);
		params.put("froumName", fourmName == null ?"1":fourmName);
		System.out.println(title);
		String result = HttpUtils.sendPostMessage(params, "UTF-8");
		log.debug("-result->>" + result);
 
	}

}